﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Net;
using System.Collections.ObjectModel;
using System.Threading;

using log4net;
using CommonLib;
using ConfigUtilLib;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;


namespace RedfinCrawlPro
{
    public class RedFinCrawler
    {
        private static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
        private static  string regionsStr = "regions";
        private static string payloadStr = "payload";
        private static  string searchResultStr = "searchResult";
        private static  string search_resultStr = "search_result";        
        private static string search_result_clustersStr = "search_result_clusters";

        private RedFinCrawler()
        {

        }

        public static object crawlForResults(FetcherConfigBaseRec fetchRec)
        {
            RedfinConfig redfinConfigRec = fetchRec as RedfinConfig;
            RawCrawledData crawledData = null;

            log.DebugFormat("    Redfin Crawler :  fetching Data from Redfin for : {0}", redfinConfigRec.FetcherName);
            
            //1. Read Fetcher  Record ,  prepare URL for Redfin 

            if (redfinConfigRec.cityName != null)
            {
                crawledData = performCityNameBasedCrawling(redfinConfigRec);
            }
            else
            {
                // Use GeoCodes 
                crawledData = performgeoPolyGonBasedCrawling(redfinConfigRec);                
            }


            return crawledData as Object;
        }

        #region PRIVATE_UTILS 

        /// <summary>
        /// City Based Query Processing ..
        /// </summary>
        /// <param name="fetchRec"></param>
        /// <returns></returns>
        private static RawCrawledData performCityNameBasedCrawling(RedfinConfig fetchRec)
        {
            string queryURL = null;
            RawCrawledData rawData = new RawCrawledData();
            //1. Read Fetcher  Record ,  prepare URL for Redfin 

            queryURL = getCityBasedRedfinURL(fetchRec.cityName);
            log.DebugFormat("Redfin City based URL to query : {0}", queryURL);

            try
            {
                
                //2.  Get JSON Response 
                WebRequest redfinCityQueryReq = WebRequest.Create(queryURL);
                WebResponse response = redfinCityQueryReq.GetResponse();
                Stream jsonStream = response.GetResponseStream();
                StreamReader jsonStreamReader = new StreamReader(response.GetResponseStream());

                string delim = "&&";
                string jsonText = jsonStreamReader.ReadToEnd();
                jsonStreamReader.Close();
                jsonText = jsonText.Remove(0, jsonText.IndexOf(delim) + delim.Length);
                JObject jsonData = JObject.Parse(jsonText);
                JToken regionsJsonBlock   = jsonData[regionsStr][0] as JToken;

                

                if (regionsJsonBlock == null)
                {
                    log.DebugFormat(" JSON Format Changed / Cannot understand format ");
                    return null;
                }

                JArray searchResultsArray = regionsJsonBlock[searchResultStr][search_resultStr] as JArray;
                JArray searchClusterArray = regionsJsonBlock[searchResultStr][search_result_clustersStr] as JArray;

                if (searchResultsArray != null)
                {
                    foreach (JObject  item in searchResultsArray)
                    {
                        RefinPropertyData propData = null;
                        log.DebugFormat(" Processing : {0}", item.ToString());
                        JObject listingItem = item["listing"] as JObject;
                        if (listingItem != null)
                        {                            
                            // Sold Property
                            propData = new RefinPropertyData(listingItem,true);                            
                        }
                        else
                        {                            
                             propData = new RefinPropertyData(item,false);
                        }
                        rawData.addPropertyToRedfinList(propData);
                    }
                }
                else if (searchClusterArray != null)
                {
                    List<RedfinPropertyClusterData> clusterList = new List<RedfinPropertyClusterData>();

                    foreach (JObject item in searchClusterArray)
                    {
                        log.DebugFormat(" CLuster Data :  {0}", item.ToString());
                        RedfinPropertyClusterData propClusterData = new RedfinPropertyClusterData(item);
                        clusterList.Add(propClusterData);                       
                    }

                    // Let us Get  Property from each Cluster 

                    foreach (RedfinPropertyClusterData dataItem in clusterList)
                    {
                        // prepate geopoint list 
                        List<GeoCoordinate> geoPointList = new List<GeoCoordinate>();
                        geoPointList.Add(new GeoCoordinate(dataItem.minLat, dataItem.minLong));
                        geoPointList.Add(new GeoCoordinate(dataItem.maxLat, dataItem.minLong));

                        geoPointList.Add(new GeoCoordinate(dataItem.maxLat, dataItem.maxLong));
                        geoPointList.Add(new GeoCoordinate(dataItem.minLat, dataItem.maxLong));
                        
                        geoPointList.Add(new GeoCoordinate(dataItem.minLat, dataItem.minLong));

                        // Perform Query 
                        performgeoPolyGonBasedCrawling(new ReadOnlyCollection<GeoCoordinate>(geoPointList), rawData);
                        // Give Some Time Spacing
                        Thread.Sleep(200);

                    }

                }

            }
            catch (Exception e)
            {
                log.DebugFormat("Error in Request Processing (URL) : {0} ", queryURL);
                
                log.DebugFormat("Exception Stack Trace : {0}", e.StackTrace);
                return null;
            }

            return rawData;


        }


        /// <summary>
        /// performgeoPolyGonBasedCrawling  
        /// </summary>
        /// <param name="fetchRec"></param>
        /// <param name="rawData"></param>
        /// <returns></returns>
        private static RawCrawledData performgeoPolyGonBasedCrawling(RedfinConfig fetchRec)
        {
            RawCrawledData rawData = new RawCrawledData();
            if (performgeoPolyGonBasedCrawling(fetchRec.PolygonGeopoints, rawData) )
            {
                return rawData;
            }
            return null;
        }


        private static bool performgeoPolyGonBasedCrawling(ReadOnlyCollection<GeoCoordinate> geoPolyGon, RawCrawledData rawData)
        {
            string queryURL = null;
            //1. Read Fetcher  Record ,  prepare URL for Redfin 

            queryURL = getGeoPolygonBasedRedfinURL(geoPolyGon);
            log.DebugFormat("Redfin Geo Polygon  based URL to query : {0}", queryURL);

            try
            {

                //2.  Get JSON Response 
                WebRequest redfingeoPolygonQueryReq = WebRequest.Create(queryURL);
                WebResponse response = redfingeoPolygonQueryReq.GetResponse();
                Stream jsonStream = response.GetResponseStream();
                StreamReader jsonStreamReader = new StreamReader(response.GetResponseStream());

                string delim = "&&";
                string jsonText = jsonStreamReader.ReadToEnd();
                jsonText = jsonText.Remove(0, jsonText.IndexOf(delim) + delim.Length);
                JObject jsonData = JObject.Parse(jsonText);

                #region JSONProcessing 
                JToken payloadJsonBlock = jsonData[payloadStr] as JToken;

                if (payloadJsonBlock == null)
                {
                    log.DebugFormat(" JSON Format Changed / Cannot understand format ");
                    return false;
                }

                JArray searchResultsArray = payloadJsonBlock[search_resultStr] as JArray;
                JArray searchClusterArray = payloadJsonBlock[search_result_clustersStr] as JArray;

                if (searchResultsArray != null)
                {
                    foreach (JObject item in searchResultsArray)
                    {
                        RefinPropertyData propData = null;
                        log.DebugFormat(" Processing : {0}", item.ToString());
                        JObject listingItem = item["listing"] as JObject;
                        if (listingItem != null)
                        {
                            // Sold Property
                            propData = new RefinPropertyData(listingItem, true);
                        }
                        else
                        {
                            log.DebugFormat(item.ToString());
                            propData = new RefinPropertyData(item, false);
                        }
                        rawData.addPropertyToRedfinList(propData);
                    }
                }
                else if (searchClusterArray != null)
                {
                    List<RedfinPropertyClusterData> clusterList = new List<RedfinPropertyClusterData>();

                    foreach (JObject item in searchClusterArray)
                    {
                        log.DebugFormat(" CLuster Data :  {0}", item.ToString());
                        RedfinPropertyClusterData propClusterData = new RedfinPropertyClusterData(item);
                        clusterList.Add(propClusterData);
                    }

                    // Let us Get  Property from each Cluster 

                    foreach (RedfinPropertyClusterData dataItem in clusterList)
                    {
                        // prepate geopoint list 
                        List<GeoCoordinate> geoPointList = new List<GeoCoordinate>();
                        geoPointList.Add(new GeoCoordinate(dataItem.minLat, dataItem.minLong));
                        geoPointList.Add(new GeoCoordinate(dataItem.maxLat, dataItem.minLong));

                        geoPointList.Add(new GeoCoordinate(dataItem.maxLat, dataItem.maxLong));
                        geoPointList.Add(new GeoCoordinate(dataItem.minLat, dataItem.maxLong));

                        geoPointList.Add(new GeoCoordinate(dataItem.minLat, dataItem.minLong));

                        // Perform Query 
                        performgeoPolyGonBasedCrawling(new ReadOnlyCollection<GeoCoordinate>(geoPointList), rawData);
                        // Give Some Time Spacing
                        Thread.Sleep(200);

                    }

                }

                jsonStreamReader.Close();

                #endregion 
            }
            catch (Exception e)
            {
                log.DebugFormat("Error in Request Processing (URL) : {0} ", queryURL);

                log.DebugFormat("Exception Stack Trace : {0}", e.StackTrace);
                return false;
            }

            return true;
        }

        /*
         * http://www.redfin.com/stingray/do/query-location?location=Redmond%2Cwa&market=seattle
         * http://www.redfin.com/stingray/do/query-location?location=Redmond%2Cwa&market=seattle
         * http://www.redfin.com/stingray/do/query-location?location=Redmond%2Cwa&market=seattle&hoa=&max_listing_approx_size=&max_num_beds=&max_parcel_size=&max_price=&max_year_built=&min_listing_approx_size=&min_parcel_size=&min_price=&min_year_built=&num_baths=&num_beds=&open_house=&pkg=-&rd=&sf=1%2C2%2C3&sold_within_days=&status=1&time_on_market_range=-&uipt=6%2C5%2C4%2C3%2C2%2C1&v=8&sp=t&al=2
         * Recommendations : http://www.redfin.com/city/neighborhood_recs?place_id=14913 
         * Geo point based query :   Higher Boundary 
         * http://www.redfin.com/stingray/do/gis-search?market=seattle&poly=-122.23147%2047.56944,-121.96917%2047.56944,-121.96917%2047.77473,-122.23147%2047.77473,-122.23147%2047.56944&hoa=&max_listing_approx_size=&max_num_beds=&max_parcel_size=&max_price=&max_year_built=&min_listing_approx_size=&min_parcel_size=&min_price=&min_year_built=&num_baths=&num_beds=&open_house=&pkg=-&rd=&sf=1%2C2%2C3&sold_within_days=&status=1&time_on_market_range=-&uipt=6%2C5%2C4%2C3%2C2%2C1&v=8&cluster_bounds=-122.2577%2047.50651,-121.94032%2047.50651,-121.94032%2047.80155,-122.2577%2047.80155,-122.2577%2047.50651&num_homes=500&sp=t&al=2
         * GeoPoint : Depth 
         * http://www.redfin.com/stingray/do/gis-search?market=seattle&poly=-122.05161%2047.66855,-122.01882%2047.66855,-122.01882%2047.6942,-122.05161%2047.6942,-122.05161%2047.66855&hoa=&max_listing_approx_size=&max_num_beds=&max_parcel_size=&max_price=&max_year_built=&min_listing_approx_size=&min_parcel_size=&min_price=&min_year_built=&num_baths=&num_beds=&open_house=&pkg=-&rd=&sf=1%2C2%2C3&sold_within_days=&status=1&time_on_market_range=-&uipt=6%2C5%2C4%2C3%2C2%2C1&v=8&cluster_bounds=-122.05489%2047.66067,-122.01522%2047.66067,-122.01522%2047.69756,-122.05489%2047.69756,-122.05489%2047.66067&num_homes=500&sp=t&al=2
         * 
         * */


        private static string citybasedQueryUrlPrefix = "http://www.redfin.com/stingray/do/query-location?";
        private static string cityUrlParam = "location={0}&al=2&";
        private static string prettymuchStatic = "hoa=&max_listing_approx_size=&max_num_beds=&max_parcel_size=&max_price=&max_year_built=&min_listing_approx_size=&min_parcel_size=&min_price=&min_year_built=&num_baths=&num_beds=&open_house=&pkg=-&rd=&sf=1%2C2%2C3%2C4&sold_within_days=90&sp=t&status=1&time_on_market_range=-&uipt=6%2C5%2C4%2C3%2C2%2C1&v=8";

        private static string getCityBasedRedfinURL( string cityname)
        {
            StringWriter completeURL = new StringWriter();
           
            completeURL.Write(citybasedQueryUrlPrefix+cityUrlParam+prettymuchStatic,cityname);            
            return completeURL.ToString();
        }


        private static string geoPolybasedQueryUrlPrefix = "http://www.redfin.com/stingray/do/gis-search?";
        private static string geoPolyUrlParam="poly={0}&al=1&";


        private static string getGeoPolygonBasedRedfinURL(ReadOnlyCollection<GeoCoordinate> geoPolyGon)
        {
            StringWriter completeURL = new StringWriter();
            StringWriter polygonStr = new StringWriter();
            bool commaSwicth = false;

            foreach (GeoCoordinate gp in geoPolyGon)
            {
                if (commaSwicth)
                {
                    polygonStr.Write(",");
                }
                else
                    commaSwicth = true;

                polygonStr.Write("{0} {1}", gp.Long, gp.Lat);
                
            }

            completeURL.Write(geoPolybasedQueryUrlPrefix + geoPolyUrlParam + prettymuchStatic, polygonStr.ToString());
            return completeURL.ToString();
        }



        #endregion 
    }
}
